#coding=utf-8
import os
import sys
import time
import json
import redis
import logging
from config import RABBITMQ_CONFIG, REDIS_SETTINGS
from RabbitmqConn import RabbitConn
from scrapy.crawler import CrawlerProcess
from shijue_auto_crawler.model import Rule
from scrapy.utils.project import get_project_settings
from shijue_auto_crawler.spiders.task import TaskSpider
'''
处理url队列, 返回该连接下所有的子url
'''
logger = logging.getLogger("TaskSpider")

def scrapy_process(rule):
    process.crawl(TaskSpider, rule)
    process.start()

def clue_id_filter(clue_id):
    """
    status = spider.server.sadd("clueidfilter", spider.clue_id)
    req_queue = spider.server.zcard(spider.name + ":requests")
    if not status and not req_queue:
        spider.logger.info("重复任务过滤 任务id:{} 任务链接:{}".format(spider.clue_id, spider.start_urls[0]))
        self.crawler.engine.close_spider(spider, 'the task has processed.')
    """
    status = redis_connector.sadd("clueidfilter", clue_id)
    req_queue = redis_connector.zcard("clue_id_"+clue_id+":requests")
    if not status and not req_queue:
        logger.info("重复任务过滤 任务id: {}".format(clue_id))
        return True
    return False

if __name__ == '__main__':
    mq_conn = RabbitConn(RABBITMQ_CONFIG, 'task')
    settings = get_project_settings()
    process = CrawlerProcess(settings)
    redis_connector = redis.StrictRedis(host=REDIS_SETTINGS['HOST'],
                                        port=REDIS_SETTINGS['PORT'],
                                        db=REDIS_SETTINGS['DB'],
                                        password=REDIS_SETTINGS['PASSWORD'])
    method_frame, header_frame, body = mq_conn.channel.basic_get(queue=mq_conn.queue)
    if body is None:
        logger.info('task队列为空，程序退出')
        time.sleep(5)
        sys.exit()
    rule_info = json.loads(body)
    clue_id = rule_info['clue_id']
    mq_conn.channel.basic_ack(delivery_tag=method_frame.delivery_tag)

    if clue_id_filter(clue_id):
        sys.exit()


    rule = Rule(rule_info)
    scrapy_process(rule)
    mq_conn.close()


